import requests
from bs4 import BeautifulSoup


def get_novel_chapters():
    root_url = 'http://www.89wx.cc/40/40293/'
    r = requests.get(root_url)
    r.encoding = 'gbk'
    soup = BeautifulSoup(r.text, "html.parser")

    data = []
    for dd in soup.find_all("dd"):
        link = dd.find('a')
        if not link:
            continue
        print(link)
        data.append((link['href'], link.get_text()))
    return data


def get_novel_content(url):
    r = requests.get(f"http://www.89wx.cc{url}")
    r.encoding = 'gbk'
    soup = BeautifulSoup(r.text, 'html.parser')
    return soup.find('div', id='content').get_text()


data = get_novel_chapters()

for url, title in data:
    print(url, title)
    with open("novels/%s.txt" % title, 'w', encoding='utf-8') as word:
        word.write(get_novel_content(url))
